{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b076bd1a-b236-4fbc-953d-8295b25122ae",
   "metadata": {},
   "source": [
    "# 🚀 GPT"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dca6836-0007-43f3-af65-d12ae1922c02",
   "metadata": {
    "tags": []
   },
   "source": [
    "In this notebook, we'll walk through the steps required to train your own GPT model on the wine review dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e6cb7c7-d3d5-4b12-b357-1f6118edffe0",
   "metadata": {},
   "source": [
    "The code is adapted from the excellent [GPT tutorial](https://keras.io/examples/generative/text_generation_with_miniature_gpt/) created by Apoorv Nandan available on the Keras website."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73350761-bef2-4e96-b3ac-a158eabd2b65",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "import json\n",
    "import re\n",
    "import string\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers, models, losses, callbacks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "339e6268-ebd7-4feb-86db-1fe7abccdbe5",
   "metadata": {},
   "source": [
    "## 0. Parameters <a name=\"parameters\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d8352af-343e-4c2e-8c91-95f8bac1c8a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_SIZE = 10000\n",
    "MAX_LEN = 80\n",
    "EMBEDDING_DIM = 256\n",
    "KEY_DIM = 256\n",
    "N_HEADS = 2\n",
    "FEED_FORWARD_DIM = 256\n",
    "VALIDATION_SPLIT = 0.2\n",
    "SEED = 42\n",
    "LOAD_MODEL = False\n",
    "BATCH_SIZE = 32\n",
    "EPOCHS = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7716fac-0010-49b0-b98e-53be2259edde",
   "metadata": {},
   "source": [
    "## 1. Load the data <a name=\"load\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93cf6b0f-9667-4146-8911-763a8a2925d3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load the full dataset\n",
    "with open(\"/app/data/wine-reviews/winemag-data-130k-v2.json\") as json_data:\n",
    "    wine_data = json.load(json_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2db5c0fc-0d5f-42ab-ade1-57e594c416ec",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "wine_data[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23a74eca-f1b7-4a46-9a1f-b5806a4ed361",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Filter the dataset\n",
    "filtered_data = [\n",
    "    \"wine review : \"\n",
    "    + x[\"country\"]\n",
    "    + \" : \"\n",
    "    + x[\"province\"]\n",
    "    + \" : \"\n",
    "    + x[\"variety\"]\n",
    "    + \" : \"\n",
    "    + x[\"description\"]\n",
    "    for x in wine_data\n",
    "    if x[\"country\"] is not None\n",
    "    and x[\"province\"] is not None\n",
    "    and x[\"variety\"] is not None\n",
    "    and x[\"description\"] is not None\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "389c20de-0422-4c48-a7b4-6ee12a7bf0e2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Count the recipes\n",
    "n_wines = len(filtered_data)\n",
    "print(f\"{n_wines} recipes loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b2e3cf7-e416-460e-874a-0dd9637bca36",
   "metadata": {},
   "outputs": [],
   "source": [
    "example = filtered_data[25]\n",
    "print(example)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f871aaf-d873-41c7-8946-e4eef7ac17c1",
   "metadata": {},
   "source": [
    "## 2. Tokenize the data <a name=\"tokenize\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b2064fb-5dcc-4657-b470-0928d10e2ddc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Pad the punctuation, to treat them as separate 'words'\n",
    "def pad_punctuation(s):\n",
    "    s = re.sub(f\"([{string.punctuation}, '\\n'])\", r\" \\1 \", s)\n",
    "    s = re.sub(\" +\", \" \", s)\n",
    "    return s\n",
    "\n",
    "\n",
    "text_data = [pad_punctuation(x) for x in filtered_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b87d7c65-9a46-492a-a5c0-a043b0d252f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display an example of a recipe\n",
    "example_data = text_data[25]\n",
    "example_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9834f916-b21a-4104-acc9-f28d3bd7a8c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert to a Tensorflow Dataset\n",
    "text_ds = (\n",
    "    tf.data.Dataset.from_tensor_slices(text_data)\n",
    "    .batch(BATCH_SIZE)\n",
    "    .shuffle(1000)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "884c0bcb-0807-45a1-8f7e-a32f2c6fa4de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a vectorisation layer\n",
    "vectorize_layer = layers.TextVectorization(\n",
    "    standardize=\"lower\",\n",
    "    max_tokens=VOCAB_SIZE,\n",
    "    output_mode=\"int\",\n",
    "    output_sequence_length=MAX_LEN + 1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d6dd34a-d905-497b-926a-405380ebcf98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adapt the layer to the training set\n",
    "vectorize_layer.adapt(text_ds)\n",
    "vocab = vectorize_layer.get_vocabulary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6c1c7ce-3cf0-40d4-a3dc-ab7090f69f2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display some token:word mappings\n",
    "for i, word in enumerate(vocab[:10]):\n",
    "    print(f\"{i}: {word}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cc30186-7ec6-4eb6-b29a-65df6714d321",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display the same example converted to ints\n",
    "example_tokenised = vectorize_layer(example_data)\n",
    "print(example_tokenised.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c195efb-84c6-4be0-a989-a7542188ad35",
   "metadata": {},
   "source": [
    "## 3. Create the Training Set <a name=\"create\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "740294a1-1a6b-4c89-92f2-036d7d1b788b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the training set of recipes and the same text shifted by one word\n",
    "def prepare_inputs(text):\n",
    "    text = tf.expand_dims(text, -1)\n",
    "    tokenized_sentences = vectorize_layer(text)\n",
    "    x = tokenized_sentences[:, :-1]\n",
    "    y = tokenized_sentences[:, 1:]\n",
    "    return x, y\n",
    "\n",
    "\n",
    "train_ds = text_ds.map(prepare_inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cad80ffb-4298-4249-86b4-9918d62534c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "example_input_output = train_ds.take(1).get_single_element()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67ff7263-f62d-44c1-997b-1aa99a393521",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example Input\n",
    "example_input_output[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef2e2cad-414c-4e6d-a2ac-6b9598f9dd01",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example Output (shifted by one token)\n",
    "example_input_output[1][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aff50401-3abe-4c10-bba8-b35bc13ad7d5",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 5. Create the causal attention mask function <a name=\"causal\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "554a4184-61c2-4eb7-a063-d965586a8188",
   "metadata": {},
   "outputs": [],
   "source": [
    "def causal_attention_mask(batch_size, n_dest, n_src, dtype):\n",
    "    i = tf.range(n_dest)[:, None]\n",
    "    j = tf.range(n_src)\n",
    "    m = i >= j - n_src + n_dest\n",
    "    mask = tf.cast(m, dtype)\n",
    "    mask = tf.reshape(mask, [1, n_dest, n_src])\n",
    "    mult = tf.concat(\n",
    "        [tf.expand_dims(batch_size, -1), tf.constant([1, 1], dtype=tf.int32)], 0\n",
    "    )\n",
    "    return tf.tile(mask, mult)\n",
    "\n",
    "\n",
    "np.transpose(causal_attention_mask(1, 10, 10, dtype=tf.int32)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3501dbad-0860-40ad-b7d6-47950e37858f",
   "metadata": {},
   "source": [
    "## 6. Create a Transformer Block layer <a name=\"transformer\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5285a1cb-fce1-46b1-b088-b596002fa9ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerBlock(layers.Layer):\n",
    "    def __init__(self, num_heads, key_dim, embed_dim, ff_dim, dropout_rate=0.1):\n",
    "        super(TransformerBlock, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.key_dim = key_dim\n",
    "        self.embed_dim = embed_dim\n",
    "        self.ff_dim = ff_dim\n",
    "        self.dropout_rate = dropout_rate\n",
    "        self.attn = layers.MultiHeadAttention(\n",
    "            num_heads, key_dim, output_shape=embed_dim\n",
    "        )\n",
    "        self.dropout_1 = layers.Dropout(self.dropout_rate)\n",
    "        self.ln_1 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.ffn_1 = layers.Dense(self.ff_dim, activation=\"relu\")\n",
    "        self.ffn_2 = layers.Dense(self.embed_dim)\n",
    "        self.dropout_2 = layers.Dropout(self.dropout_rate)\n",
    "        self.ln_2 = layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        input_shape = tf.shape(inputs)\n",
    "        batch_size = input_shape[0]\n",
    "        seq_len = input_shape[1]\n",
    "        causal_mask = causal_attention_mask(\n",
    "            batch_size, seq_len, seq_len, tf.bool\n",
    "        )\n",
    "        attention_output, attention_scores = self.attn(\n",
    "            inputs,\n",
    "            inputs,\n",
    "            attention_mask=causal_mask,\n",
    "            return_attention_scores=True,\n",
    "        )\n",
    "        attention_output = self.dropout_1(attention_output)\n",
    "        out1 = self.ln_1(inputs + attention_output)\n",
    "        ffn_1 = self.ffn_1(out1)\n",
    "        ffn_2 = self.ffn_2(ffn_1)\n",
    "        ffn_output = self.dropout_2(ffn_2)\n",
    "        return (self.ln_2(out1 + ffn_output), attention_scores)\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super().get_config()\n",
    "        config.update(\n",
    "            {\n",
    "                \"key_dim\": self.key_dim,\n",
    "                \"embed_dim\": self.embed_dim,\n",
    "                \"num_heads\": self.num_heads,\n",
    "                \"ff_dim\": self.ff_dim,\n",
    "                \"dropout_rate\": self.dropout_rate,\n",
    "            }\n",
    "        )\n",
    "        return config"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "076a6be0-9796-4974-9bcd-6ebbcfe7514e",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 7. Create the Token and Position Embedding <a name=\"embedder\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdf5cb25-88ae-4026-9e21-c1e6b5094a2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TokenAndPositionEmbedding(layers.Layer):\n",
    "    def __init__(self, max_len, vocab_size, embed_dim):\n",
    "        super(TokenAndPositionEmbedding, self).__init__()\n",
    "        self.max_len = max_len\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embed_dim = embed_dim\n",
    "        self.token_emb = layers.Embedding(\n",
    "            input_dim=vocab_size, output_dim=embed_dim\n",
    "        )\n",
    "        self.pos_emb = layers.Embedding(input_dim=max_len, output_dim=embed_dim)\n",
    "\n",
    "    def call(self, x):\n",
    "        maxlen = tf.shape(x)[-1]\n",
    "        positions = tf.range(start=0, limit=maxlen, delta=1)\n",
    "        positions = self.pos_emb(positions)\n",
    "        x = self.token_emb(x)\n",
    "        return x + positions\n",
    "\n",
    "    def get_config(self):\n",
    "        config = super().get_config()\n",
    "        config.update(\n",
    "            {\n",
    "                \"max_len\": self.max_len,\n",
    "                \"vocab_size\": self.vocab_size,\n",
    "                \"embed_dim\": self.embed_dim,\n",
    "            }\n",
    "        )\n",
    "        return config"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aac2e2d4-5980-47e3-b5b0-6c41c0c2d152",
   "metadata": {},
   "source": [
    "## 8. Build the Transformer model <a name=\"transformer_decoder\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c57596e-e17d-4959-b6e8-7581b0bace3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = layers.Input(shape=(None,), dtype=tf.int32)\n",
    "x = TokenAndPositionEmbedding(MAX_LEN, VOCAB_SIZE, EMBEDDING_DIM)(inputs)\n",
    "x, attention_scores = TransformerBlock(\n",
    "    N_HEADS, KEY_DIM, EMBEDDING_DIM, FEED_FORWARD_DIM\n",
    ")(x)\n",
    "outputs = layers.Dense(VOCAB_SIZE, activation=\"softmax\")(x)\n",
    "gpt = models.Model(inputs=inputs, outputs=[outputs, attention_scores])\n",
    "gpt.compile(\"adam\", loss=[losses.SparseCategoricalCrossentropy(), None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a1c3b0f-3382-444d-bb04-bae143ae5d61",
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "800a3c6e-fb11-4792-b6bc-9a43a7c977ad",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if LOAD_MODEL:\n",
    "    # model.load_weights('./models/model')\n",
    "    gpt = models.load_model(\"./models/gpt\", compile=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35b14665-4359-447b-be58-3fd58ba69084",
   "metadata": {},
   "source": [
    "## 9. Train the Transformer <a name=\"train\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ddcff5f-829d-4449-99d2-9a3cb68f7d72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a TextGenerator checkpoint\n",
    "class TextGenerator(callbacks.Callback):\n",
    "    def __init__(self, index_to_word, top_k=10):\n",
    "        self.index_to_word = index_to_word\n",
    "        self.word_to_index = {\n",
    "            word: index for index, word in enumerate(index_to_word)\n",
    "        }\n",
    "\n",
    "    def sample_from(self, probs, temperature):\n",
    "        probs = probs ** (1 / temperature)\n",
    "        probs = probs / np.sum(probs)\n",
    "        return np.random.choice(len(probs), p=probs), probs\n",
    "\n",
    "    def generate(self, start_prompt, max_tokens, temperature):\n",
    "        start_tokens = [\n",
    "            self.word_to_index.get(x, 1) for x in start_prompt.split()\n",
    "        ]\n",
    "        sample_token = None\n",
    "        info = []\n",
    "        while len(start_tokens) < max_tokens and sample_token != 0:\n",
    "            x = np.array([start_tokens])\n",
    "            y, att = self.model.predict(x, verbose=0)\n",
    "            sample_token, probs = self.sample_from(y[0][-1], temperature)\n",
    "            info.append(\n",
    "                {\n",
    "                    \"prompt\": start_prompt,\n",
    "                    \"word_probs\": probs,\n",
    "                    \"atts\": att[0, :, -1, :],\n",
    "                }\n",
    "            )\n",
    "            start_tokens.append(sample_token)\n",
    "            start_prompt = start_prompt + \" \" + self.index_to_word[sample_token]\n",
    "        print(f\"\\ngenerated text:\\n{start_prompt}\\n\")\n",
    "        return info\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        self.generate(\"wine review\", max_tokens=80, temperature=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "349865fe-ffbe-450e-97be-043ae1740e78",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a model save checkpoint\n",
    "model_checkpoint_callback = callbacks.ModelCheckpoint(\n",
    "    filepath=\"./checkpoint/checkpoint.ckpt\",\n",
    "    save_weights_only=True,\n",
    "    save_freq=\"epoch\",\n",
    "    verbose=0,\n",
    ")\n",
    "\n",
    "tensorboard_callback = callbacks.TensorBoard(log_dir=\"./logs\")\n",
    "\n",
    "# Tokenize starting prompt\n",
    "text_generator = TextGenerator(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "461c2b3e-b5ae-4def-8bd9-e7bab8c63d8e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "gpt.fit(\n",
    "    train_ds,\n",
    "    epochs=EPOCHS,\n",
    "    callbacks=[model_checkpoint_callback, tensorboard_callback, text_generator],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "369bde44-2e39-4bc6-8549-a3a27ecce55c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Save the final model\n",
    "gpt.save(\"./models/gpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d64e02d2-84dc-40c8-8446-40c09adf1e20",
   "metadata": {},
   "source": [
    "# 3. Generate text using the Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ad23adb-3ec9-4e9a-9a59-b9f9bafca649",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_probs(info, vocab, top_k=5):\n",
    "    for i in info:\n",
    "        highlighted_text = []\n",
    "        for word, att_score in zip(\n",
    "            i[\"prompt\"].split(), np.mean(i[\"atts\"], axis=0)\n",
    "        ):\n",
    "            highlighted_text.append(\n",
    "                '<span style=\"background-color:rgba(135,206,250,'\n",
    "                + str(att_score / max(np.mean(i[\"atts\"], axis=0)))\n",
    "                + ');\">'\n",
    "                + word\n",
    "                + \"</span>\"\n",
    "            )\n",
    "        highlighted_text = \" \".join(highlighted_text)\n",
    "        display(HTML(highlighted_text))\n",
    "\n",
    "        word_probs = i[\"word_probs\"]\n",
    "        p_sorted = np.sort(word_probs)[::-1][:top_k]\n",
    "        i_sorted = np.argsort(word_probs)[::-1][:top_k]\n",
    "        for p, i in zip(p_sorted, i_sorted):\n",
    "            print(f\"{vocab[i]}:   \\t{np.round(100*p,2)}%\")\n",
    "        print(\"--------\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cf25578-d47c-4b26-8252-fcdf2316a4ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "info = text_generator.generate(\n",
    "    \"wine review : us\", max_tokens=80, temperature=1.0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ae2da8e-9b7c-4b71-b37b-021115b3d7ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "info = text_generator.generate(\n",
    "    \"wine review : italy\", max_tokens=80, temperature=0.5\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cae6d5d-263d-4455-b96c-f315cbe284ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "info = text_generator.generate(\n",
    "    \"wine review : germany\", max_tokens=80, temperature=0.5\n",
    ")\n",
    "print_probs(info, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcc7a7c0-6c64-47d3-acee-96e48f025e01",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4122154c-9e41-44ba-8c5f-fe88a55fe977",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
